
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>3.6 糗事百科网站爬虫 · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    
    <link rel="prev" href="Json与JsonPath.html" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    前言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="正则表达式.html">
            
                <a href="正则表达式.html">
            
                    
                    第一章：正则表达式
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="正则表达式入门.html">
            
                <a href="正则表达式入门.html">
            
                    
                    1.1 正则表达式入门
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="Python中的正则表达式.html">
            
                <a href="Python中的正则表达式.html">
            
                    
                    1.2 Python中的正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="基本正则表达式.html">
            
                <a href="基本正则表达式.html">
            
                    
                    1.3 基本正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.4" data-path="分组.html">
            
                <a href="分组.html">
            
                    
                    1.4 分组
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.5" data-path="先行断言.html">
            
                <a href="先行断言.html">
            
                    
                    1.5 先行断言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.6" data-path="标记.html">
            
                <a href="标记.html">
            
                    
                    1.6 标记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.7" data-path="替换与正则表达式编译.html">
            
                <a href="替换与正则表达式编译.html">
            
                    
                    1.7 替换与正则编译
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.8" data-path="正则表达式练习.html">
            
                <a href="正则表达式练习.html">
            
                    
                    1.8 正则表达式练习
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="Python爬虫学习目标.html">
            
                <a href="Python爬虫学习目标.html">
            
                    
                    第二章：爬虫与Requests
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="Requests的简单使用.html">
            
                <a href="Requests的简单使用.html">
            
                    
                    2.1 Requests的简单使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="深入了解Requests.html">
            
                <a href="深入了解Requests.html">
            
                    
                    2.2 深入了解Requets
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="结构化与非结构化.html">
            
                <a href="结构化与非结构化.html">
            
                    
                    第三章：非结构化与结构化数据提取
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="正则抓取.html">
            
                <a href="正则抓取.html">
            
                    
                    3.1 使用正则的爬虫
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="xpath与XML.html">
            
                <a href="xpath与XML.html">
            
                    
                    3.2 Xpath与XML类库
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="lxml.html">
            
                <a href="lxml.html">
            
                    
                    3.3 LXML库的使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.4" data-path="Xpath爬虫案例.html">
            
                <a href="Xpath爬虫案例.html">
            
                    
                    3.4 案例：贴吧图片
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.5" data-path="Json与JsonPath.html">
            
                <a href="Json与JsonPath.html">
            
                    
                    3.5 Json与Json数据获取
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="1.4.6" data-path="实例：糗事百科网站抓取.html">
            
                <a href="实例：糗事百科网站抓取.html">
            
                    
                    3.6 糗事百科网站爬虫
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >3.6 糗事百科网站爬虫</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <h2 id="&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x722C;&#x866B;">&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x722C;&#x866B;</h2>
<p>&#x722C;&#x53D6;&#x7CD7;&#x4E8B;&#x767E;&#x79D1;&#x6BB5;&#x5B50;&#xFF0C;&#x5047;&#x8BBE;&#x9875;&#x9762;&#x7684;URL&#x662F; <a href="http://www.qiushibaike.com/8hr/page/1" target="_blank">http://www.qiushibaike.com/8hr/page/1</a></p>
<p>&#x8981;&#x6C42;&#xFF1A;</p>
<ol>
<li><p>&#x4F7F;&#x7528;<code>requests</code>&#x83B7;&#x53D6;&#x9875;&#x9762;&#x4FE1;&#x606F;&#xFF0C;&#x7528;<code>XPath / re</code>&#x505A;&#x6570;&#x636E;&#x63D0;&#x53D6;</p>
</li>
<li><p>&#x83B7;&#x53D6;&#x6BCF;&#x4E2A;&#x5E16;&#x5B50;&#x91CC;&#x7684;<code>&#x7528;&#x6237;&#x5934;&#x50CF;&#x94FE;&#x63A5;</code>&#x3001;<code>&#x7528;&#x6237;&#x59D3;&#x540D;</code>&#x3001;<code>&#x6BB5;&#x5B50;&#x5185;&#x5BB9;</code>&#x3001;<code>&#x70B9;&#x8D5E;&#x6B21;&#x6570;</code>&#x548C;<code>&#x8BC4;&#x8BBA;&#x6B21;&#x6570;</code></p>
</li>
<li><p>&#x4FDD;&#x5B58;&#x5230;<code>json</code>&#x6587;&#x4EF6;&#x5185;</p>
</li>
</ol>
<h3 id="&#x53C2;&#x8003;&#x4EE3;&#x7801;">&#x53C2;&#x8003;&#x4EE3;&#x7801;</h3>
<pre><code class="lang-python"><span class="hljs-comment">#coding=utf-8</span>
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> retrying <span class="hljs-keyword">import</span> retry
<span class="hljs-keyword">from</span> lxml <span class="hljs-keyword">import</span> etree

<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Qiubai_spider</span><span class="hljs-params">()</span>:</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self)</span>:</span>
        self.url = <span class="hljs-string">&quot;http://www.qiushibaike.com/8hr/page/{}/&quot;</span>
        self.headers = {
            <span class="hljs-string">&quot;User-Agent&quot;</span>:<span class="hljs-string">&quot;Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1 Trident/5.0;&quot;</span>
        }

<span class="hljs-meta">    @retry(stop_max_attempt_number=5) #&#x8C03;&#x7528;retry&#xFF0C;&#x5F53;assert&#x51FA;&#x9519;&#x65F6;&#x5019;&#xFF0C;&#x91CD;&#x590D;&#x8BF7;&#x6C42;5&#x6B21;</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse_url</span><span class="hljs-params">(self,url)</span>:</span>
        response = requests.get(url,timeout=<span class="hljs-number">10</span>,headers=self.headers) <span class="hljs-comment">#&#x8BF7;&#x6C42;url</span>
        <span class="hljs-keyword">assert</span> response.status_code==<span class="hljs-number">200</span>  <span class="hljs-comment">#&#x5F53;&#x54CD;&#x5E94;&#x7801;&#x4E0D;&#x662F;200&#x65F6;&#x5019;&#xFF0C;&#x505A;&#x65AD;&#x8A00;&#x62A5;&#x9519;&#x5904;&#x7406;</span>
        print(url)
        <span class="hljs-keyword">return</span> etree.HTML(response.text) <span class="hljs-comment">#&#x8FD4;&#x56DE;etree&#x4E4B;&#x540E;&#x7684;html</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse_content</span><span class="hljs-params">(self,html)</span>:</span>
        item_temp = html.xpath(<span class="hljs-string">&quot;//div[@class=&apos;article block untagged mb15&apos;]&quot;</span>)
        print(len(item_temp))
        <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> item_temp:
            <span class="hljs-comment">#&#x83B7;&#x53D6;&#x7528;&#x6237;&#x5934;&#x50CF;&#x5730;&#x5740;</span>
            avatar = item.xpath(<span class="hljs-string">&quot;./div[1]/a[1]/img/@src&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-keyword">if</span> len(item.xpath(<span class="hljs-string">&quot;./div[1]/a[1]/img/@src&quot;</span>))&gt;<span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-keyword">None</span>
            <span class="hljs-comment">#&#x4E3A;&#x5934;&#x50CF;&#x5730;&#x5740;&#x6DFB;&#x52A0;&#x524D;&#x7F00;</span>
            <span class="hljs-keyword">if</span> avatar <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">None</span> <span class="hljs-keyword">and</span> <span class="hljs-keyword">not</span> avatar.startswith(<span class="hljs-string">&quot;http:&quot;</span>):
                avatar = <span class="hljs-string">&quot;http:&quot;</span>+avatar
            print(avatar)
            name = item.xpath(<span class="hljs-string">&quot;./div[1]/a[2]/h2/text()&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-comment">#&#x83B7;&#x53D6;&#x7528;&#x6237;&#x540D;</span>
            print(name)
            content = item.xpath(<span class="hljs-string">&quot;./a[@class=&apos;contentHerf&apos;]/div/span/text()&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-comment">#&#x83B7;&#x53D6;&#x5185;&#x5BB9;</span>
            print(content)
            star_number = item.xpath(<span class="hljs-string">&quot;./div[@class=&apos;stats&apos;]/span[1]/i/text()&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-comment">#&#x83B7;&#x53D6;&#x70B9;&#x8D5E;&#x6570;</span>
            print(star_number)
            comment_number = item.xpath(<span class="hljs-string">&quot;./div[@class=&apos;stats&apos;]/span[2]/a/i/text()&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-comment">#&#x83B7;&#x53D6;&#x8BC4;&#x8BBA;&#x6570;</span>
            print(comment_number)
            print(<span class="hljs-string">&quot;*&quot;</span>*<span class="hljs-number">100</span>)

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x51FD;&#x6570;&#x7684;&#x4E3B;&#x8981;&#x903B;&#x8F91;&#x5B9E;&#x73B0;
        &apos;&apos;&apos;</span>
        url = self.url.format(<span class="hljs-number">1</span>) <span class="hljs-comment">#&#x83B7;&#x53D6;&#x5230;url</span>
        html = self.parse_url(url) <span class="hljs-comment">#&#x8BF7;&#x6C42;url</span>
        self.parse_content(html) <span class="hljs-comment">#&#x89E3;&#x6790;&#x9875;&#x9762;&#x5185;&#x5BB9;&#x5E76;&#x628A;&#x5185;&#x5BB9;&#x5B58;&#x5165;&#x5185;&#x5BB9;&#x961F;&#x5217;</span>

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    qiubai = Qiubai_spider()
    qiubai.run()
</code></pre>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="Json与JsonPath.html" class="navigation navigation-prev navigation-unique" aria-label="Previous page: 3.5 Json与Json数据获取">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"3.6 糗事百科网站爬虫","level":"1.4.6","depth":2,"previous":{"title":"3.5 Json与Json数据获取","level":"1.4.5","depth":2,"path":"notes/Json与JsonPath.md","ref":"notes/Json与JsonPath.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"notes/实例：糗事百科网站抓取.md","mtime":"2019-05-04T08:39:48.019Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-08T02:50:45.307Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

